{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package loaded\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Logistic_regression with MNIST\n",
    "author hadxu(hadxu123@gmail.com)\n",
    "\"\"\"\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import matplotlib.pyplot as plt\n",
    "print 'Package loaded'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "#Download and extract MNIST dataset\n",
    "mnist = input_data.read_data_sets('data/',one_hot=True)\n",
    "trainimg = mnist.train.images\n",
    "trainlabel = mnist.train.labels\n",
    "testimg = mnist.test.images\n",
    "testlabel = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#parameters of Logistic Regression\n",
    "learning_rate = 0.01\n",
    "training_epochs = 50\n",
    "batch_size = 100\n",
    "display_step = 1\n",
    "x = tf.placeholder('float',[None,28*28])\n",
    "y = tf.placeholder('float',[None,10])\n",
    "W = tf.Variable(tf.zeros([784,10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "activation = tf.nn.softmax(tf.matmul(x,W)+b)\n",
    "#cost = tf.reduce_mean(-tf.reduce_sum(y*tf.log(activation),reduction_indices=1))\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(y*tf.log(activation) + (1-y)*tf.log(1-activation), reduction_indices=1)) # Cross entropy\n",
    "#cost = tf.reduce_mean(tf.pow(activation-y, 2))\n",
    "\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n",
    "correct_prediction = tf.equal(tf.argmax(activation,1),tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,'float'))\n",
    "init = tf.initialize_all_variables()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/050 cost: 1.691291186 train_acc: 0.870\n",
      "Epoch: 001/050 cost: 0.962134818 train_acc: 0.930\n",
      "Epoch: 002/050 cost: 0.821396246 train_acc: 0.820\n",
      "Epoch: 003/050 cost: 0.744501618 train_acc: 0.880\n",
      "Epoch: 004/050 cost: 0.705570070 train_acc: 0.930\n",
      "Epoch: 005/050 cost: 0.669074314 train_acc: 0.920\n",
      "Epoch: 006/050 cost: 0.644957957 train_acc: 0.910\n",
      "Epoch: 007/050 cost: 0.624491437 train_acc: 0.850\n",
      "Epoch: 008/050 cost: 0.613271614 train_acc: 0.920\n",
      "Epoch: 009/050 cost: 0.605349815 train_acc: 0.880\n",
      "Epoch: 010/050 cost: 0.601586389 train_acc: 0.900\n",
      "Epoch: 011/050 cost: 0.585578671 train_acc: 0.950\n",
      "Epoch: 012/050 cost: 0.576798616 train_acc: 0.940\n",
      "Epoch: 013/050 cost: 0.564583284 train_acc: 0.950\n",
      "Epoch: 014/050 cost: 0.568582169 train_acc: 0.920\n",
      "Epoch: 015/050 cost: 0.553210066 train_acc: 0.870\n",
      "Epoch: 016/050 cost: 0.554388077 train_acc: 0.890\n",
      "Epoch: 017/050 cost: 0.542737691 train_acc: 0.840\n",
      "Epoch: 018/050 cost: 0.541170811 train_acc: 0.910\n",
      "Epoch: 019/050 cost: 0.538855565 train_acc: 0.910\n",
      "Epoch: 020/050 cost: 0.538254621 train_acc: 0.920\n",
      "Epoch: 021/050 cost: 0.530835912 train_acc: 0.960\n",
      "Epoch: 022/050 cost: 0.522037844 train_acc: 0.920\n",
      "Epoch: 023/050 cost: 0.530215086 train_acc: 0.930\n",
      "Epoch: 024/050 cost: 0.517315367 train_acc: 0.920\n",
      "Epoch: 025/050 cost: 0.517525984 train_acc: 0.880\n",
      "Epoch: 026/050 cost: 0.516480477 train_acc: 0.920\n",
      "Epoch: 027/050 cost: 0.512451837 train_acc: 0.920\n",
      "Epoch: 028/050 cost: 0.514584421 train_acc: 0.890\n",
      "Epoch: 029/050 cost: 0.508348630 train_acc: 0.920\n",
      "Epoch: 030/050 cost: 0.501783607 train_acc: 0.900\n",
      "Epoch: 031/050 cost: 0.506124154 train_acc: 0.950\n",
      "Epoch: 032/050 cost: 0.503609273 train_acc: 0.950\n",
      "Epoch: 033/050 cost: 0.494374885 train_acc: 0.920\n",
      "Epoch: 034/050 cost: 0.505061827 train_acc: 0.900\n",
      "Epoch: 035/050 cost: 0.499845274 train_acc: 0.880\n",
      "Epoch: 036/050 cost: 0.508708407 train_acc: 0.930\n",
      "Epoch: 037/050 cost: 0.496467121 train_acc: 0.900\n",
      "Epoch: 038/050 cost: 0.490741922 train_acc: 0.930\n",
      "Epoch: 039/050 cost: 0.495485547 train_acc: 0.850\n",
      "Epoch: 040/050 cost: 0.482892403 train_acc: 0.930\n",
      "Epoch: 041/050 cost: 0.492758499 train_acc: 0.900\n",
      "Epoch: 042/050 cost: 0.483924227 train_acc: 0.910\n",
      "Epoch: 043/050 cost: 0.495801120 train_acc: 0.880\n",
      "Epoch: 044/050 cost: 0.482731858 train_acc: 0.890\n",
      "Epoch: 045/050 cost: 0.493482322 train_acc: 0.940\n",
      "Epoch: 046/050 cost: 0.485849597 train_acc: 0.950\n",
      "Epoch: 047/050 cost: 0.479545757 train_acc: 0.870\n",
      "Epoch: 048/050 cost: 0.489440552 train_acc: 0.980\n",
      "Epoch: 049/050 cost: 0.478663260 train_acc: 0.940\n",
      "Optimization Finished!\n",
      "('Accuracy:', 0.92400002)\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    # Training cycle\n",
    "    for epoch in range(training_epochs):\n",
    "        avg_cost = 0.\n",
    "        num_batch = int(mnist.train.num_examples/batch_size)\n",
    "        # Loop over all batches\n",
    "        for i in range(num_batch): \n",
    "            if 0: # Using tensorflow API\n",
    "                batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            else: # Random batch sampling \n",
    "                randidx = np.random.randint(trainimg.shape[0], size=batch_size)\n",
    "                batch_xs = trainimg[randidx, :]\n",
    "                batch_ys = trainlabel[randidx, :]                \n",
    "            \n",
    "            # Fit training using batch data\n",
    "            sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys})\n",
    "            # Compute average loss\n",
    "            avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys})/num_batch\n",
    "            \n",
    "        # Display logs per epoch step\n",
    "        if epoch % display_step == 0:\n",
    "            train_acc = accuracy.eval({x: batch_xs, y: batch_ys})\n",
    "            print (\"Epoch: %03d/%03d cost: %.9f train_acc: %.3f\" \n",
    "                   % (epoch, training_epochs, avg_cost, train_acc))\n",
    "\n",
    "    print (\"Optimization Finished!\")\n",
    "\n",
    "    # Test model\n",
    "    # Calculate accuracy\n",
    "    print (\"Accuracy:\", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))\n",
    "\n",
    "print (\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
